library(here)
here() starts at /Users/maarten/Documents/projects/gamified-feedback-study
library(dplyr)
Attaching package: 'dplyr'
The following objects are masked from 'package:stats':
filter, lag
The following objects are masked from 'package:base':
intersect, setdiff, setequal, union
library(ggplot2)
library(scales)
library(patchwork)
Warning: package 'patchwork' was built under R version 4.3.3
library(stringr)
library(tidyr)
library(lme4)
Loading required package: Matrix
Attaching package: 'Matrix'
The following objects are masked from 'package:tidyr':
expand, pack, unpack
library(lmerTest)
Attaching package: 'lmerTest'
The following object is masked from 'package:lme4':
lmer
The following object is masked from 'package:stats':
step
library(brms)
Loading required package: Rcpp
Loading 'brms' package (version 2.21.0). Useful instructions
can be found by typing help('brms'). A more detailed introduction
to the package is available through vignette('brms_overview').
Attaching package: 'brms'
The following object is masked from 'package:lme4':
ngrps
The following object is masked from 'package:stats':
ar
# Set up parallel processing for Bayesian models
library(future)
plan(multisession, workers = 4)
Helper functions for plots and tables:
source(here("scripts", "00_visualisation_functions.R"))
Loading required package: data.table
Attaching package: 'data.table'
The following objects are masked from 'package:dplyr':
between, first, last
Loading required package: flextable
Load processed data:
d_test <- readRDS(here("data", "processed", "d_test.rds"))
add_experiment_cols <- function (data) {
data |>
mutate(exp_order = case_when(
gamified_first == 0 & exp_group == "score" ~ "Control—Score",
gamified_first == 0 & exp_group == "both" ~ "Control—Both",
gamified_first == 1 & exp_group == "score" ~ "Score—Control",
gamified_first == 1 & exp_group == "both" ~ "Both—Control"
)) |>
mutate(type = ifelse(gamified, "Gamified", "Control"))
}
Helper function for interpreting Bayes factors:
bf_to_strength <- function (bf) {
direction <- "for"
if (bf < 1) {
bf <- 1/bf
direction <- "against"
}
strength <- case_when(
bf == 1 ~ "No",
bf < 3 ~ "Anecdotal",
bf < 10 ~ "Moderate",
bf < 30 ~ "Strong",
bf < 100 ~ "Very strong",
TRUE ~ "Extreme"
)
paste0(strength, " evidence ", direction)
}
d_test_acc <- d_test |>
group_by(subject, exp_group, block, condition, gamified, gamified_first) |>
summarise(accuracy = mean(correct))
`summarise()` has grouped output by 'subject', 'exp_group', 'block',
'condition', 'gamified'. You can override using the `.groups` argument.
d_test_acc_agg <- d_test_acc |>
group_by(block, condition, gamified, gamified_first, exp_group) |>
summarise(acc = mean(accuracy, na.rm = T),
acc_se = sd(accuracy, na.rm = T)/sqrt(n())) |>
ungroup() |>
add_experiment_cols()
`summarise()` has grouped output by 'block', 'condition', 'gamified',
'gamified_first'. You can override using the `.groups` argument.
p_test_acc <- plot_data(d_test_acc_agg, acc, acc_se, "Accuracy") +
scale_y_continuous(limits = c(.35, .6), labels = scales::percent_format())
p_test_acc
Prepare data for modelling by mean-centering categorical predictors:
d_test_m <- d_test |>
mutate(exp_group_c = ifelse(exp_group == "score", 0, 1),
exp_group_c = exp_group_c - mean(exp_group_c),
gamified_first_c = gamified_first - mean(gamified_first))
m_test_acc <- glmer(correct ~ gamified +
gamified:exp_group_c +
gamified:gamified_first_c +
gamified:gamified_first_c:exp_group_c +
(1 | subject) + (1 | fact),
family = "binomial",
data = d_test_m)
summary(m_test_acc)
Generalized linear mixed model fit by maximum likelihood (Laplace
Approximation) [glmerMod]
Family: binomial ( logit )
Formula:
correct ~ gamified + gamified:exp_group_c + gamified:gamified_first_c +
gamified:gamified_first_c:exp_group_c + (1 | subject) + (1 | fact)
Data: d_test_m
AIC BIC logLik deviance df.resid
8909.2 8977.8 -4444.6 8889.2 7014
Scaled residuals:
Min 1Q Median 3Q Max
-3.8290 -0.8456 0.3436 0.8197 3.1357
Random effects:
Groups Name Variance Std.Dev.
subject (Intercept) 0.6635 0.8145
fact (Intercept) 0.3379 0.5813
Number of obs: 7024, groups: subject, 152; fact, 70
Fixed effects:
Estimate Std. Error z value
(Intercept) -0.015175 0.103341 -0.147
gamifiedTRUE 0.008417 0.052769 0.160
gamifiedFALSE:exp_group_c -0.153904 0.153331 -1.004
gamifiedTRUE:exp_group_c -0.178017 0.153562 -1.159
gamifiedFALSE:gamified_first_c 0.170561 0.153567 1.111
gamifiedTRUE:gamified_first_c -0.208889 0.153591 -1.360
gamifiedFALSE:exp_group_c:gamified_first_c -0.166474 0.308267 -0.540
gamifiedTRUE:exp_group_c:gamified_first_c -0.206697 0.308355 -0.670
Pr(>|z|)
(Intercept) 0.883
gamifiedTRUE 0.873
gamifiedFALSE:exp_group_c 0.316
gamifiedTRUE:exp_group_c 0.246
gamifiedFALSE:gamified_first_c 0.267
gamifiedTRUE:gamified_first_c 0.174
gamifiedFALSE:exp_group_c:gamified_first_c 0.589
gamifiedTRUE:exp_group_c:gamified_first_c 0.503
Correlation of Fixed Effects:
(Intr) gmTRUE gmfdFALSE:x__ gmfdTRUE:x__ gmfdFALSE:g__
gamifidTRUE -0.255
gmfdFALSE:x__ -0.019 0.004
gmfdTRUE:x__ -0.017 0.005 0.760
gmfdFALSE:g__ -0.019 0.007 -0.009 -0.007
gmfdTRUE:g__ -0.015 0.012 -0.007 -0.004 0.763
gFALSE:__:_ -0.007 0.001 -0.028 -0.022 -0.025
gTRUE:__:__ -0.006 0.006 -0.022 -0.013 -0.023
gmfdTRUE:g__ gFALSE:__:
gamifidTRUE
gmfdFALSE:x__
gmfdTRUE:x__
gmfdFALSE:g__
gmfdTRUE:g__
gFALSE:__:_ -0.023
gTRUE:__:__ -0.019 0.761
print_model_table(m_test_acc)
Effect | Estimate | SE | z-value | p-value |
|---|---|---|---|---|
Intercept (Control) | -0.02 | 0.10 | -0.15 | .883 |
Gamification | 0.01 | 0.05 | 0.16 | .873 |
Type of gamification (Points-and-progress - Points) | -0.18 | 0.15 | -1.16 | .246 |
Type of gamification on Control (Points-and-progress - Points) | -0.15 | 0.15 | -1.00 | .316 |
Block on gamification (1 - 2) | -0.21 | 0.15 | -1.36 | .174 |
Block on Control (2 - 1) | 0.17 | 0.15 | 1.11 | .267 |
Type of gamification; block difference (Block 1 - Block 2) | -0.21 | 0.31 | -0.67 | .503 |
Type of gamification on Control; block difference (Block 2 - Block 1) | -0.17 | 0.31 | -0.54 | .589 |
p_test_acc_m <- plot_model_fit(m_test_acc, d_test_m, y_lab = "Accuracy") +
scale_y_continuous(limits = c(.35, .6), labels = scales::percent_format(accuracy = .1))
block condition gamified gamified_first exp_group gamified_first_c
1 1 Control FALSE FALSE both -0.5380125
2 1 Control FALSE FALSE score -0.5380125
3 1 Points TRUE TRUE score 0.4619875
4 1 Points-and-progress TRUE TRUE both 0.4619875
5 2 Control FALSE TRUE both 0.4619875
6 2 Control FALSE TRUE score 0.4619875
7 2 Points TRUE FALSE score -0.5380125
8 2 Points-and-progress TRUE FALSE both -0.5380125
exp_group_c pred_val exp_order type
1 0.5545273 0.4644061 Control—Both Control
2 -0.4454727 0.4804404 Control—Score Control
3 -0.4454727 0.5046444 Score—Control Gamified
4 0.5545273 0.4366104 Both—Control Gamified
5 0.5545273 0.4839130 Both—Control Control
6 -0.4454727 0.5415149 Score—Control Control
7 -0.4454727 0.5337956 Control—Score Gamified
8 0.5545273 0.5171377 Control—Both Gamified
Scale for y is already present.
Adding another scale for y, which will replace the existing scale.
p_test_acc_m
Fit again with brms so that we can calculate Bayes
Factors. Because we expect any fixed effects to be at most moderate in
size, we will use a weakly informative Normal(0, 1) prior for these
effects.
m_test_acc_brms <- brm(correct ~ gamified +
gamified:exp_group_c +
gamified:gamified_first_c +
gamified:gamified_first_c:exp_group_c +
(1 | subject) + (1 | fact),
family = "bernoulli",
data = d_test_m,
prior = set_prior("normal(0, 1)", class = "b"),
chains = 4,
iter = 11000,
warmup = 1000,
sample_prior = TRUE,
future = TRUE,
seed = 0)
Compiling Stan program...
Trying to compile a simple C file
Running /Library/Frameworks/R.framework/Resources/bin/R CMD SHLIB foo.c
using C compiler: ‘Apple clang version 15.0.0 (clang-1500.3.9.4)’
using SDK: ‘MacOSX14.4.sdk’
clang -arch arm64 -I"/Library/Frameworks/R.framework/Resources/include" -DNDEBUG -I"/Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/Rcpp/include/" -I"/Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/RcppEigen/include/" -I"/Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/RcppEigen/include/unsupported" -I"/Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/BH/include" -I"/Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/StanHeaders/include/src/" -I"/Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/StanHeaders/include/" -I"/Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/RcppParallel/include/" -I"/Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/rstan/include" -DEIGEN_NO_DEBUG -DBOOST_DISABLE_ASSERTS -DBOOST_PENDING_INTEGER_LOG2_HPP -DSTAN_THREADS -DUSE_STANC3 -DSTRICT_R_HEADERS -DBOOST_PHOENIX_NO_VARIADIC_EXPRESSION -D_HAS_AUTO_PTR_ETC=0 -include '/Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/StanHeaders/include/stan/math/prim/fun/Eigen.hpp' -D_REENTRANT -DRCPP_PARALLEL_USE_TBB=1 -I/opt/R/arm64/include -fPIC -falign-functions=64 -Wall -g -O2 -c foo.c -o foo.o
In file included from <built-in>:1:
In file included from /Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/StanHeaders/include/stan/math/prim/fun/Eigen.hpp:22:
In file included from /Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/RcppEigen/include/Eigen/Dense:1:
In file included from /Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/RcppEigen/include/Eigen/Core:19:
/Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/RcppEigen/include/Eigen/src/Core/util/Macros.h:679:10: fatal error: 'cmath' file not found
#include <cmath>
^~~~~~~
1 error generated.
make[1]: *** [foo.o] Error 1
Start sampling
Loading required package: StanHeaders
rstan version 2.32.6 (Stan version 2.32.2)
For execution on a local, multicore CPU with excess RAM we recommend calling
options(mc.cores = parallel::detectCores()).
To avoid recompilation of unchanged Stan programs, we recommend calling
rstan_options(auto_write = TRUE)
For within-chain threading using `reduce_sum()` or `map_rect()` Stan functions,
change `threads_per_chain` option:
rstan_options(threads_per_chain = 1)
Attaching package: 'rstan'
The following object is masked from 'package:tidyr':
extract
SAMPLING FOR MODEL 'anon_model' NOW (CHAIN 1).
Chain 1:
Chain 1: Gradient evaluation took 0.000451 seconds
Chain 1: 1000 transitions using 10 leapfrog steps per transition would take 4.51 seconds.
Chain 1: Adjust your expectations accordingly!
Chain 1:
Chain 1:
Chain 1: Iteration: 1 / 11000 [ 0%] (Warmup)
Chain 1: Iteration: 1001 / 11000 [ 9%] (Sampling)
Chain 1: Iteration: 2100 / 11000 [ 19%] (Sampling)
Chain 1: Iteration: 3200 / 11000 [ 29%] (Sampling)
Chain 1: Iteration: 4300 / 11000 [ 39%] (Sampling)
Chain 1: Iteration: 5400 / 11000 [ 49%] (Sampling)
Chain 1: Iteration: 6500 / 11000 [ 59%] (Sampling)
Chain 1: Iteration: 7600 / 11000 [ 69%] (Sampling)
Chain 1: Iteration: 8700 / 11000 [ 79%] (Sampling)
Chain 1: Iteration: 9800 / 11000 [ 89%] (Sampling)
Chain 1: Iteration: 10900 / 11000 [ 99%] (Sampling)
Chain 1: Iteration: 11000 / 11000 [100%] (Sampling)
Chain 1:
Chain 1: Elapsed Time: 6.898 seconds (Warm-up)
Chain 1: 47.705 seconds (Sampling)
Chain 1: 54.603 seconds (Total)
Chain 1:
SAMPLING FOR MODEL 'anon_model' NOW (CHAIN 2).
Chain 2:
Chain 2: Gradient evaluation took 0.000921 seconds
Chain 2: 1000 transitions using 10 leapfrog steps per transition would take 9.21 seconds.
Chain 2: Adjust your expectations accordingly!
Chain 2:
Chain 2:
Chain 2: Iteration: 1 / 11000 [ 0%] (Warmup)
Chain 2: Iteration: 1001 / 11000 [ 9%] (Sampling)
Chain 2: Iteration: 2100 / 11000 [ 19%] (Sampling)
Chain 2: Iteration: 3200 / 11000 [ 29%] (Sampling)
Chain 2: Iteration: 4300 / 11000 [ 39%] (Sampling)
Chain 2: Iteration: 5400 / 11000 [ 49%] (Sampling)
Chain 2: Iteration: 6500 / 11000 [ 59%] (Sampling)
Chain 2: Iteration: 7600 / 11000 [ 69%] (Sampling)
Chain 2: Iteration: 8700 / 11000 [ 79%] (Sampling)
Chain 2: Iteration: 9800 / 11000 [ 89%] (Sampling)
Chain 2: Iteration: 10900 / 11000 [ 99%] (Sampling)
Chain 2: Iteration: 11000 / 11000 [100%] (Sampling)
Chain 2:
Chain 2: Elapsed Time: 7.442 seconds (Warm-up)
Chain 2: 46.282 seconds (Sampling)
Chain 2: 53.724 seconds (Total)
Chain 2:
SAMPLING FOR MODEL 'anon_model' NOW (CHAIN 3).
Chain 3:
Chain 3: Gradient evaluation took 0.000796 seconds
Chain 3: 1000 transitions using 10 leapfrog steps per transition would take 7.96 seconds.
Chain 3: Adjust your expectations accordingly!
Chain 3:
Chain 3:
Chain 3: Iteration: 1 / 11000 [ 0%] (Warmup)
Chain 3: Iteration: 1001 / 11000 [ 9%] (Sampling)
Chain 3: Iteration: 2100 / 11000 [ 19%] (Sampling)
Chain 3: Iteration: 3200 / 11000 [ 29%] (Sampling)
Chain 3: Iteration: 4300 / 11000 [ 39%] (Sampling)
Chain 3: Iteration: 5400 / 11000 [ 49%] (Sampling)
Chain 3: Iteration: 6500 / 11000 [ 59%] (Sampling)
Chain 3: Iteration: 7600 / 11000 [ 69%] (Sampling)
Chain 3: Iteration: 8700 / 11000 [ 79%] (Sampling)
Chain 3: Iteration: 9800 / 11000 [ 89%] (Sampling)
Chain 3: Iteration: 10900 / 11000 [ 99%] (Sampling)
Chain 3: Iteration: 11000 / 11000 [100%] (Sampling)
Chain 3:
Chain 3: Elapsed Time: 8.04 seconds (Warm-up)
Chain 3: 45.446 seconds (Sampling)
Chain 3: 53.486 seconds (Total)
Chain 3:
SAMPLING FOR MODEL 'anon_model' NOW (CHAIN 4).
Chain 4:
Chain 4: Gradient evaluation took 0.007346 seconds
Chain 4: 1000 transitions using 10 leapfrog steps per transition would take 73.46 seconds.
Chain 4: Adjust your expectations accordingly!
Chain 4:
Chain 4:
Chain 4: Iteration: 1 / 11000 [ 0%] (Warmup)
Chain 4: Iteration: 1001 / 11000 [ 9%] (Sampling)
Chain 4: Iteration: 2100 / 11000 [ 19%] (Sampling)
Chain 4: Iteration: 3200 / 11000 [ 29%] (Sampling)
Chain 4: Iteration: 4300 / 11000 [ 39%] (Sampling)
Chain 4: Iteration: 5400 / 11000 [ 49%] (Sampling)
Chain 4: Iteration: 6500 / 11000 [ 59%] (Sampling)
Chain 4: Iteration: 7600 / 11000 [ 69%] (Sampling)
Chain 4: Iteration: 8700 / 11000 [ 79%] (Sampling)
Chain 4: Iteration: 9800 / 11000 [ 89%] (Sampling)
Chain 4: Iteration: 10900 / 11000 [ 99%] (Sampling)
Chain 4: Iteration: 11000 / 11000 [100%] (Sampling)
Chain 4:
Chain 4: Elapsed Time: 7.152 seconds (Warm-up)
Chain 4: 44.654 seconds (Sampling)
Chain 4: 51.806 seconds (Total)
Chain 4:
summary(m_test_acc_brms)
Family: bernoulli
Links: mu = logit
Formula: correct ~ gamified + gamified:exp_group_c + gamified:gamified_first_c + gamified:gamified_first_c:exp_group_c + (1 | subject) + (1 | fact)
Data: d_test_m (Number of observations: 7024)
Draws: 4 chains, each with iter = 11000; warmup = 1000; thin = 1;
total post-warmup draws = 40000
Multilevel Hyperparameters:
~fact (Number of levels: 70)
Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
sd(Intercept) 0.60 0.06 0.49 0.72 1.00 9887 18255
~subject (Number of levels: 152)
Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
sd(Intercept) 0.84 0.06 0.72 0.97 1.00 10168 18058
Regression Coefficients:
Estimate Est.Error l-95% CI u-95% CI
Intercept -0.02 0.11 -0.22 0.19
gamifiedTRUE 0.01 0.05 -0.10 0.11
gamifiedFALSE:exp_group_c -0.15 0.15 -0.45 0.15
gamifiedTRUE:exp_group_c -0.17 0.15 -0.48 0.13
gamifiedFALSE:gamified_first_c 0.17 0.15 -0.13 0.47
gamifiedTRUE:gamified_first_c -0.21 0.15 -0.51 0.10
gamifiedFALSE:exp_group_c:gamified_first_c -0.14 0.29 -0.72 0.44
gamifiedTRUE:exp_group_c:gamified_first_c -0.18 0.29 -0.76 0.40
Rhat Bulk_ESS Tail_ESS
Intercept 1.00 6025 11946
gamifiedTRUE 1.00 56931 30687
gamifiedFALSE:exp_group_c 1.00 6665 13012
gamifiedTRUE:exp_group_c 1.00 6751 13442
gamifiedFALSE:gamified_first_c 1.00 6871 14131
gamifiedTRUE:gamified_first_c 1.00 6895 14193
gamifiedFALSE:exp_group_c:gamified_first_c 1.00 7791 15057
gamifiedTRUE:exp_group_c:gamified_first_c 1.00 7762 14611
Draws were sampled using sampling(NUTS). For each parameter, Bulk_ESS
and Tail_ESS are effective sample size measures, and Rhat is the potential
scale reduction factor on split chains (at convergence, Rhat = 1).
Inspect the posterior sample distributions of the fixed effects:
plot(m_test_acc_brms, nvariables = 8, variable = "^b_", regex = TRUE)
Do a hypothesis test for all fixed-effect coefficients (both main
effects and interactions) in the model being equal to zero. The column
Evid.Ratio shows the Bayes Factor in favour of the null
hypothesis (\(BF_{01}\)).
h_test_acc <- hypothesis(m_test_acc_brms,
c("gamifiedTRUE = 0",
"gamifiedFALSE:exp_group_c = 0",
"gamifiedTRUE:exp_group_c = 0",
"gamifiedFALSE:gamified_first_c = 0",
"gamifiedTRUE:gamified_first_c = 0",
"gamifiedFALSE:exp_group_c:gamified_first_c = 0",
"gamifiedTRUE:exp_group_c:gamified_first_c = 0"),
class = "b")
h_test_acc$hypothesis |>
mutate(BF10 = 1 / Evid.Ratio,
evidence_for_null = sapply(Evid.Ratio, bf_to_strength))
This hypothesis test is calculating the Savage-Dickey density ratio at zero, which is a ratio of the posterior density at zero relative to the prior density at zero (indicated by dashed vertical line). Values above 1 indicate a stronger belief that the effect is indeed zero after having observed the data.
sd_ratio_acc <- plot(h_test_acc, nvariables = 8, variable = "^b_", regex = TRUE, plot = FALSE)
sd_ratio_acc[[1]] +
geom_vline(xintercept = 0, linetype = "dashed", colour = "grey25")
The Bayesian model finds anecdotal to strong evidence in favour of the null hypothesis (no effect on posttest accuracy) for each of the regression coefficients.
Response time on correct answers only.
To keep the visualisation of average response times by condition simple, we calculate the median RT per participant, and then take the mean and SD of these medians (which are themselves roughly normally distributed).
d_test_rt <- d_test |>
filter(correct) |>
mutate(rt = rt / 1000) |>
group_by(subject, exp_group, block, condition, gamified, gamified_first) |>
summarise(rt_median = median(rt, na.rm = TRUE))
`summarise()` has grouped output by 'subject', 'exp_group', 'block',
'condition', 'gamified'. You can override using the `.groups` argument.
d_test_rt_agg <- d_test_rt |>
group_by(block, condition, gamified, gamified_first, exp_group) |>
summarise(rt_mean = mean(rt_median, na.rm = T),
rt_se = sd(rt_median, na.rm = T)/sqrt(n())) |>
ungroup() |>
add_experiment_cols()
`summarise()` has grouped output by 'block', 'condition', 'gamified',
'gamified_first'. You can override using the `.groups` argument.
p_test_rt <- plot_data(d_test_rt_agg, rt_mean, rt_se, "Response time (s)") +
scale_y_continuous(limits = c(3, 6), labels = scales::comma_format())
p_test_rt
Since RT data is not normally distributed, we fit a lognormal model to the response times. (See https://bbolker.github.io/mixedmodels-misc/glmmFAQ.html#gamma-glmms .) Prepare data for modelling by mean-centering categorical predictors:
d_test_rt_m <- d_test |>
filter(correct) |>
mutate(log_rt = log(rt / 1000)) |>
mutate(exp_group_c = ifelse(exp_group == "score", 0, 1),
exp_group_c = exp_group_c - mean(exp_group_c),
gamified_first_c = gamified_first - mean(gamified_first))
m_test_rt <- lmer(log_rt ~ gamified +
gamified:exp_group_c +
gamified:gamified_first_c +
gamified:gamified_first_c:exp_group_c +
(1 | subject) + (1 | fact),
data = d_test_rt_m)
summary(m_test_rt)
Linear mixed model fit by REML. t-tests use Satterthwaite's method [
lmerModLmerTest]
Formula:
log_rt ~ gamified + gamified:exp_group_c + gamified:gamified_first_c +
gamified:gamified_first_c:exp_group_c + (1 | subject) + (1 | fact)
Data: d_test_rt_m
REML criterion at convergence: 5381.8
Scaled residuals:
Min 1Q Median 3Q Max
-2.9689 -0.6786 -0.1696 0.5261 4.8408
Random effects:
Groups Name Variance Std.Dev.
subject (Intercept) 0.07237 0.2690
fact (Intercept) 0.01968 0.1403
Residual 0.22400 0.4733
Number of obs: 3673, groups: subject, 152; fact, 70
Fixed effects:
Estimate Std. Error df
(Intercept) 1.463e+00 3.028e-02 2.065e+02
gamifiedTRUE 7.858e-03 1.611e-02 3.515e+03
gamifiedFALSE:exp_group_c 1.367e-02 5.045e-02 1.720e+02
gamifiedTRUE:exp_group_c 4.380e-02 5.067e-02 1.736e+02
gamifiedFALSE:gamified_first_c 6.011e-02 5.053e-02 1.708e+02
gamifiedTRUE:gamified_first_c 1.174e-01 5.064e-02 1.711e+02
gamifiedFALSE:exp_group_c:gamified_first_c 2.316e-01 1.015e-01 1.723e+02
gamifiedTRUE:exp_group_c:gamified_first_c 1.980e-01 1.017e-01 1.727e+02
t value Pr(>|t|)
(Intercept) 48.315 <2e-16 ***
gamifiedTRUE 0.488 0.6257
gamifiedFALSE:exp_group_c 0.271 0.7867
gamifiedTRUE:exp_group_c 0.864 0.3885
gamifiedFALSE:gamified_first_c 1.190 0.2358
gamifiedTRUE:gamified_first_c 2.318 0.0216 *
gamifiedFALSE:exp_group_c:gamified_first_c 2.282 0.0237 *
gamifiedTRUE:exp_group_c:gamified_first_c 1.946 0.0533 .
---
Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
Correlation of Fixed Effects:
(Intr) gmTRUE gmfdFALSE:x__ gmfdTRUE:x__ gmfdFALSE:g__
gamifidTRUE -0.262
gmfdFALSE:x__ -0.040 0.000
gmfdTRUE:x__ -0.040 0.006 0.792
gmfdFALSE:g__ -0.028 0.018 -0.005 -0.016
gmfdTRUE:g__ -0.018 0.018 -0.016 0.002 0.796
gFALSE:__:_ -0.004 -0.016 -0.036 -0.028 -0.048
gTRUE:__:__ -0.014 0.029 -0.027 -0.008 -0.045
gmfdTRUE:g__ gFALSE:__:
gamifidTRUE
gmfdFALSE:x__
gmfdTRUE:x__
gmfdFALSE:g__
gmfdTRUE:g__
gFALSE:__:_ -0.046
gTRUE:__:__ -0.046 0.792
print_model_table(m_test_rt)
Effect | Estimate | SE | df | t-value | p-value |
|---|---|---|---|---|---|
Intercept (Control) | 1.463 | 0.030 | 206.46 | 48.32 | <.001 |
Gamification | 0.008 | 0.016 | 3,514.91 | 0.49 | .626 |
Type of gamification (Points-and-progress - Points) | 0.044 | 0.051 | 173.57 | 0.86 | .389 |
Type of gamification on Control (Points-and-progress - Points) | 0.014 | 0.050 | 171.97 | 0.27 | .787 |
Block on gamification (1 - 2) | 0.117 | 0.051 | 171.14 | 2.32 | .022 |
Block on Control (2 - 1) | 0.060 | 0.051 | 170.84 | 1.19 | .236 |
Type of gamification; block difference (Block 1 - Block 2) | 0.198 | 0.102 | 172.68 | 1.95 | .053 |
Type of gamification on Control; block difference (Block 2 - Block 1) | 0.232 | 0.101 | 172.31 | 2.28 | .024 |
d_model_fit <- crossing(
gamified = TRUE,
exp_group_c = 0,
gamified_first_c = sort(unique(d_test_rt_m$gamified_first_c))
)
d_model_fit$model_fit <- predict(m_test_rt,
newdata = d_model_fit,
re.form = NA,
type = "response") |>
exp() # Transform logRT to RT
d_model_fit
d_model_fit <- crossing(
gamified = FALSE,
exp_group_c = sort(unique(d_test_rt_m$exp_group_c)),
gamified_first_c = sort(unique(d_test_rt_m$gamified_first_c))
)
d_model_fit$model_fit <- predict(m_test_rt,
newdata = d_model_fit,
re.form = NA,
type = "response") |>
exp() # Transform logRT to RT
d_model_fit
p_test_rt_m <- plot_model_fit(m_test_rt, d_test_rt_m, exp_trans = TRUE, y_lab = "Response time (s)") +
scale_y_continuous(limits = c(3, 6), labels = scales::comma_format())
block condition gamified gamified_first exp_group gamified_first_c
1 1 Control FALSE FALSE both -0.5363463
2 1 Control FALSE FALSE score -0.5363463
3 1 Points TRUE TRUE score 0.4636537
4 1 Points-and-progress TRUE TRUE both 0.4636537
5 2 Control FALSE TRUE both 0.4636537
6 2 Control FALSE TRUE score 0.4636537
7 2 Points TRUE FALSE score -0.5363463
8 2 Points-and-progress TRUE FALSE both -0.5363463
exp_group_c pred_val exp_order type
1 0.5722842 3.924932 Control—Both Control
2 -0.4277158 4.383624 Control—Score Control
3 -0.4277158 4.336746 Score—Control Gamified
4 0.5722842 4.966467 Both—Control Gamified
5 0.5722842 4.758713 Both—Control Control
6 -0.4277158 4.216226 Score—Control Control
7 -0.4277158 4.197234 Control—Score Gamified
8 0.5722842 3.943450 Control—Both Gamified
Scale for y is already present.
Adding another scale for y, which will replace the existing scale.
p_test_rt_m
Fit again with brms so that we can calculate Bayes
Factors. Because we expect any fixed effects to be at most moderate in
size, we will use a weakly informative Normal(0, 1) prior for these
effects.
m_test_rt_brms <- brm(log_rt ~ gamified +
gamified:exp_group_c +
gamified:gamified_first_c +
gamified:gamified_first_c:exp_group_c +
(1 | subject) + (1 | fact),
family = "gaussian",
data = d_test_rt_m,
prior = set_prior("normal(0, .1)", class = "b"),
chains = 4,
iter = 11000,
warmup = 1000,
sample_prior = TRUE,
future = TRUE,
seed = 0)
Compiling Stan program...
Trying to compile a simple C file
Running /Library/Frameworks/R.framework/Resources/bin/R CMD SHLIB foo.c
using C compiler: ‘Apple clang version 15.0.0 (clang-1500.3.9.4)’
using SDK: ‘MacOSX14.4.sdk’
clang -arch arm64 -I"/Library/Frameworks/R.framework/Resources/include" -DNDEBUG -I"/Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/Rcpp/include/" -I"/Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/RcppEigen/include/" -I"/Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/RcppEigen/include/unsupported" -I"/Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/BH/include" -I"/Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/StanHeaders/include/src/" -I"/Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/StanHeaders/include/" -I"/Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/RcppParallel/include/" -I"/Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/rstan/include" -DEIGEN_NO_DEBUG -DBOOST_DISABLE_ASSERTS -DBOOST_PENDING_INTEGER_LOG2_HPP -DSTAN_THREADS -DUSE_STANC3 -DSTRICT_R_HEADERS -DBOOST_PHOENIX_NO_VARIADIC_EXPRESSION -D_HAS_AUTO_PTR_ETC=0 -include '/Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/StanHeaders/include/stan/math/prim/fun/Eigen.hpp' -D_REENTRANT -DRCPP_PARALLEL_USE_TBB=1 -I/opt/R/arm64/include -fPIC -falign-functions=64 -Wall -g -O2 -c foo.c -o foo.o
In file included from <built-in>:1:
In file included from /Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/StanHeaders/include/stan/math/prim/fun/Eigen.hpp:22:
In file included from /Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/RcppEigen/include/Eigen/Dense:1:
In file included from /Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/RcppEigen/include/Eigen/Core:19:
/Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library/RcppEigen/include/Eigen/src/Core/util/Macros.h:679:10: fatal error: 'cmath' file not found
#include <cmath>
^~~~~~~
1 error generated.
make[1]: *** [foo.o] Error 1
Start sampling
SAMPLING FOR MODEL 'anon_model' NOW (CHAIN 1).
Chain 1:
Chain 1: Gradient evaluation took 0.00026 seconds
Chain 1: 1000 transitions using 10 leapfrog steps per transition would take 2.6 seconds.
Chain 1: Adjust your expectations accordingly!
Chain 1:
Chain 1:
Chain 1: Iteration: 1 / 11000 [ 0%] (Warmup)
Chain 1: Iteration: 1001 / 11000 [ 9%] (Sampling)
Chain 1: Iteration: 2100 / 11000 [ 19%] (Sampling)
Chain 1: Iteration: 3200 / 11000 [ 29%] (Sampling)
Chain 1: Iteration: 4300 / 11000 [ 39%] (Sampling)
Chain 1: Iteration: 5400 / 11000 [ 49%] (Sampling)
Chain 1: Iteration: 6500 / 11000 [ 59%] (Sampling)
Chain 1: Iteration: 7600 / 11000 [ 69%] (Sampling)
Chain 1: Iteration: 8700 / 11000 [ 79%] (Sampling)
Chain 1: Iteration: 9800 / 11000 [ 89%] (Sampling)
Chain 1: Iteration: 10900 / 11000 [ 99%] (Sampling)
Chain 1: Iteration: 11000 / 11000 [100%] (Sampling)
Chain 1:
Chain 1: Elapsed Time: 5.082 seconds (Warm-up)
Chain 1: 19.351 seconds (Sampling)
Chain 1: 24.433 seconds (Total)
Chain 1:
SAMPLING FOR MODEL 'anon_model' NOW (CHAIN 2).
Chain 2:
Chain 2: Gradient evaluation took 0.000251 seconds
Chain 2: 1000 transitions using 10 leapfrog steps per transition would take 2.51 seconds.
Chain 2: Adjust your expectations accordingly!
Chain 2:
Chain 2:
Chain 2: Iteration: 1 / 11000 [ 0%] (Warmup)
Chain 2: Iteration: 1001 / 11000 [ 9%] (Sampling)
Chain 2: Iteration: 2100 / 11000 [ 19%] (Sampling)
Chain 2: Iteration: 3200 / 11000 [ 29%] (Sampling)
Chain 2: Iteration: 4300 / 11000 [ 39%] (Sampling)
Chain 2: Iteration: 5400 / 11000 [ 49%] (Sampling)
Chain 2: Iteration: 6500 / 11000 [ 59%] (Sampling)
Chain 2: Iteration: 7600 / 11000 [ 69%] (Sampling)
Chain 2: Iteration: 8700 / 11000 [ 79%] (Sampling)
Chain 2: Iteration: 9800 / 11000 [ 89%] (Sampling)
Chain 2: Iteration: 10900 / 11000 [ 99%] (Sampling)
Chain 2: Iteration: 11000 / 11000 [100%] (Sampling)
Chain 2:
Chain 2: Elapsed Time: 5.168 seconds (Warm-up)
Chain 2: 22.738 seconds (Sampling)
Chain 2: 27.906 seconds (Total)
Chain 2:
SAMPLING FOR MODEL 'anon_model' NOW (CHAIN 3).
Chain 3:
Chain 3: Gradient evaluation took 0.003372 seconds
Chain 3: 1000 transitions using 10 leapfrog steps per transition would take 33.72 seconds.
Chain 3: Adjust your expectations accordingly!
Chain 3:
Chain 3:
Chain 3: Iteration: 1 / 11000 [ 0%] (Warmup)
Chain 3: Iteration: 1001 / 11000 [ 9%] (Sampling)
Chain 3: Iteration: 2100 / 11000 [ 19%] (Sampling)
Chain 3: Iteration: 3200 / 11000 [ 29%] (Sampling)
Chain 3: Iteration: 4300 / 11000 [ 39%] (Sampling)
Chain 3: Iteration: 5400 / 11000 [ 49%] (Sampling)
Chain 3: Iteration: 6500 / 11000 [ 59%] (Sampling)
Chain 3: Iteration: 7600 / 11000 [ 69%] (Sampling)
Chain 3: Iteration: 8700 / 11000 [ 79%] (Sampling)
Chain 3: Iteration: 9800 / 11000 [ 89%] (Sampling)
Chain 3: Iteration: 10900 / 11000 [ 99%] (Sampling)
Chain 3: Iteration: 11000 / 11000 [100%] (Sampling)
Chain 3:
Chain 3: Elapsed Time: 5.154 seconds (Warm-up)
Chain 3: 26.32 seconds (Sampling)
Chain 3: 31.474 seconds (Total)
Chain 3:
SAMPLING FOR MODEL 'anon_model' NOW (CHAIN 4).
Chain 4:
Chain 4: Gradient evaluation took 0.000179 seconds
Chain 4: 1000 transitions using 10 leapfrog steps per transition would take 1.79 seconds.
Chain 4: Adjust your expectations accordingly!
Chain 4:
Chain 4:
Chain 4: Iteration: 1 / 11000 [ 0%] (Warmup)
Chain 4: Iteration: 1001 / 11000 [ 9%] (Sampling)
Chain 4: Iteration: 2100 / 11000 [ 19%] (Sampling)
Chain 4: Iteration: 3200 / 11000 [ 29%] (Sampling)
Chain 4: Iteration: 4300 / 11000 [ 39%] (Sampling)
Chain 4: Iteration: 5400 / 11000 [ 49%] (Sampling)
Chain 4: Iteration: 6500 / 11000 [ 59%] (Sampling)
Chain 4: Iteration: 7600 / 11000 [ 69%] (Sampling)
Chain 4: Iteration: 8700 / 11000 [ 79%] (Sampling)
Chain 4: Iteration: 9800 / 11000 [ 89%] (Sampling)
Chain 4: Iteration: 10900 / 11000 [ 99%] (Sampling)
Chain 4: Iteration: 11000 / 11000 [100%] (Sampling)
Chain 4:
Chain 4: Elapsed Time: 5.124 seconds (Warm-up)
Chain 4: 19.236 seconds (Sampling)
Chain 4: 24.36 seconds (Total)
Chain 4:
summary(m_test_rt_brms)
Family: gaussian
Links: mu = identity; sigma = identity
Formula: log_rt ~ gamified + gamified:exp_group_c + gamified:gamified_first_c + gamified:gamified_first_c:exp_group_c + (1 | subject) + (1 | fact)
Data: d_test_rt_m (Number of observations: 3673)
Draws: 4 chains, each with iter = 11000; warmup = 1000; thin = 1;
total post-warmup draws = 40000
Multilevel Hyperparameters:
~fact (Number of levels: 70)
Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
sd(Intercept) 0.14 0.02 0.12 0.18 1.00 10785 18637
~subject (Number of levels: 152)
Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
sd(Intercept) 0.27 0.02 0.24 0.31 1.00 7496 13982
Regression Coefficients:
Estimate Est.Error l-95% CI u-95% CI
Intercept 1.46 0.03 1.40 1.53
gamifiedTRUE 0.01 0.02 -0.02 0.04
gamifiedFALSE:exp_group_c 0.01 0.04 -0.08 0.09
gamifiedTRUE:exp_group_c 0.03 0.04 -0.05 0.12
gamifiedFALSE:gamified_first_c 0.04 0.04 -0.05 0.12
gamifiedTRUE:gamified_first_c 0.09 0.04 0.01 0.17
gamifiedFALSE:exp_group_c:gamified_first_c 0.09 0.06 -0.04 0.22
gamifiedTRUE:exp_group_c:gamified_first_c 0.06 0.07 -0.07 0.19
Rhat Bulk_ESS Tail_ESS
Intercept 1.00 5821 10898
gamifiedTRUE 1.00 44637 31503
gamifiedFALSE:exp_group_c 1.00 7546 14751
gamifiedTRUE:exp_group_c 1.00 7609 15045
gamifiedFALSE:gamified_first_c 1.00 6795 14173
gamifiedTRUE:gamified_first_c 1.00 6834 13035
gamifiedFALSE:exp_group_c:gamified_first_c 1.00 12457 22090
gamifiedTRUE:exp_group_c:gamified_first_c 1.00 12397 21890
Further Distributional Parameters:
Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
sigma 0.47 0.01 0.46 0.48 1.00 45311 30732
Draws were sampled using sampling(NUTS). For each parameter, Bulk_ESS
and Tail_ESS are effective sample size measures, and Rhat is the potential
scale reduction factor on split chains (at convergence, Rhat = 1).
Inspect the posterior sample distributions of the fixed effects:
plot(m_test_rt_brms, nvariables = 8, variable = "^b_", regex = TRUE)
Do a hypothesis test for all fixed-effect coefficients (both main
effects and interactions) in the model being equal to zero. The column
Evid.Ratio shows the Bayes Factor in favour of the null
hypothesis (\(BF_{01}\)).
h_test_rt <- hypothesis(m_test_rt_brms,
c("gamifiedTRUE = 0",
"gamifiedFALSE:exp_group_c = 0",
"gamifiedTRUE:exp_group_c = 0",
"gamifiedFALSE:gamified_first_c = 0",
"gamifiedTRUE:gamified_first_c = 0",
"gamifiedFALSE:exp_group_c:gamified_first_c = 0",
"gamifiedTRUE:exp_group_c:gamified_first_c = 0"),
class = "b")
h_test_rt$hypothesis |>
mutate(BF10 = 1 / Evid.Ratio,
evidence_for_null = sapply(Evid.Ratio, bf_to_strength))
This hypothesis test is calculating the Savage-Dickey density ratio at zero, which is a ratio of the posterior density at zero relative to the prior density at zero (indicated by dashed vertical line). Values above 1 indicate a stronger belief that the effect is indeed zero after having observed the data.
sd_ratio_rt <- plot(h_test_rt, nvariables = 8, variable = "^b_", regex = TRUE, plot = FALSE)
sd_ratio_rt[[1]] +
geom_vline(xintercept = 0, linetype = "dashed", colour = "grey25")
The Bayesian model finds evidence in favour of the null hypothesis
(no effect on posttest accuracy) for all but two of the regression
coefficients. There is moderate evidence for an order effect on the
gamified conditions, as well as weak anecdotal evidence in favour of an
indirect order effect of gamification type in the control condition
((gamifiedFALSE:exp_group_c:gamified_first_c)).
d_model_fit <- crossing(
gamified = TRUE,
exp_group_c = 0,
gamified_first_c = sort(unique(d_test_rt_m$gamified_first_c))
)
d_model_fit$model_fit <- predict(m_test_rt_brms,
newdata = d_model_fit,
re_formula = NA)[,1] |>
exp() # Transform logRT to RT
d_model_fit
d_model_fit <- crossing(
gamified = FALSE,
exp_group_c = sort(unique(d_test_rt_m$exp_group_c)),
gamified_first_c = sort(unique(d_test_rt_m$gamified_first_c))
)
d_model_fit$model_fit <- predict(m_test_rt_brms,
newdata = d_model_fit,
re_formula = NA)[,1] |>
exp() # Transform logRT to RT
d_model_fit
(p_test_acc | p_test_rt) +
plot_layout(guides = "collect") +
plot_annotation(tag_levels = "a")
ggsave(here("output", "test_performance.png"), width = 7.5, height = 3)
fwrite(h_test_acc$hypothesis, here("output", "hypothesis_tests", "h_test_acc.csv"))
fwrite(h_test_rt$hypothesis, here("output", "hypothesis_tests", "h_test_rt.csv"))
sessionInfo()
R version 4.3.1 (2023-06-16)
Platform: aarch64-apple-darwin20 (64-bit)
Running under: macOS Sonoma 14.5
Matrix products: default
BLAS: /Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/lib/libRblas.0.dylib
LAPACK: /Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/lib/libRlapack.dylib; LAPACK version 3.11.0
locale:
[1] en_US.UTF-8/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8
time zone: Europe/Amsterdam
tzcode source: internal
attached base packages:
[1] stats graphics grDevices utils datasets methods base
other attached packages:
[1] rstan_2.32.6 StanHeaders_2.32.6 flextable_0.9.6 data.table_1.14.8
[5] future_1.33.0 brms_2.21.0 Rcpp_1.0.11 lmerTest_3.1-3
[9] lme4_1.1-34 Matrix_1.6-1.1 tidyr_1.3.0 stringr_1.5.0
[13] patchwork_1.3.0 scales_1.3.0 ggplot2_3.5.0 dplyr_1.1.3
[17] here_1.0.1
loaded via a namespace (and not attached):
[1] gridExtra_2.3 inline_0.3.19 sandwich_3.0-2
[4] rlang_1.1.1 magrittr_2.0.3 multcomp_1.4-25
[7] matrixStats_1.3.0 compiler_4.3.1 loo_2.7.0
[10] reshape2_1.4.4 systemfonts_1.0.4 callr_3.7.3
[13] vctrs_0.6.3 httpcode_0.3.0 pkgconfig_2.0.3
[16] crayon_1.5.2 fastmap_1.1.1 ellipsis_0.3.2
[19] backports_1.4.1 labeling_0.4.3 utf8_1.2.3
[22] promises_1.2.1 rmarkdown_2.25 ps_1.7.5
[25] nloptr_2.0.3 ragg_1.2.7 purrr_1.0.2
[28] xfun_0.40 cachem_1.0.8 jsonlite_1.8.7
[31] later_1.3.1 uuid_1.2-0 parallel_4.3.1
[34] prettyunits_1.2.0 R6_2.5.1 bslib_0.5.1
[37] stringi_1.7.12 parallelly_1.36.0 boot_1.3-28.1
[40] jquerylib_0.1.4 numDeriv_2016.8-1.1 estimability_1.4.1
[43] knitr_1.44 zoo_1.8-12 bayesplot_1.11.1
[46] httpuv_1.6.12 splines_4.3.1 tidyselect_1.2.0
[49] rstudioapi_0.15.0 abind_1.4-5 yaml_2.3.7
[52] codetools_0.2-19 curl_5.1.0 processx_3.8.2
[55] listenv_0.9.0 pkgbuild_1.4.2 plyr_1.8.9
[58] lattice_0.21-9 tibble_3.2.1 shiny_1.8.0
[61] withr_2.5.1 bridgesampling_1.1-2 askpass_1.2.0
[64] posterior_1.5.0 coda_0.19-4 evaluate_0.22
[67] survival_3.5-7 RcppParallel_5.1.7 zip_2.3.1
[70] xml2_1.3.5 pillar_1.9.0 tensorA_0.36.2.1
[73] checkmate_2.3.1 stats4_4.3.1 distributional_0.4.0
[76] generics_0.1.3 rprojroot_2.0.3 rstantools_2.4.0
[79] munsell_0.5.0 minqa_1.2.6 globals_0.16.2
[82] xtable_1.8-4 glue_1.6.2 gdtools_0.3.7
[85] emmeans_1.8.9 tools_4.3.1 gfonts_0.2.0
[88] mvtnorm_1.2-3 grid_4.3.1 QuickJSR_1.1.3
[91] colorspace_2.1-0 nlme_3.1-163 cli_3.6.1
[94] textshaping_0.3.7 officer_0.6.6 fontBitstreamVera_0.1.1
[97] fansi_1.0.4 Brobdingnag_1.2-9 V8_4.3.3
[100] gtable_0.3.4 sass_0.4.7 digest_0.6.33
[103] fontquiver_0.2.1 crul_1.4.2 TH.data_1.1-2
[106] farver_2.1.1 htmltools_0.5.6 lifecycle_1.0.3
[109] mime_0.12 openssl_2.1.1 fontLiberation_0.1.0
[112] MASS_7.3-60